{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "885df2d1",
   "metadata": {},
   "source": [
    "## Milvus没有使用docker的方式测试"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d26bc733",
   "metadata": {},
   "source": [
    "### 导入模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a47abf4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymilvus import MilvusClient\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c8a6417",
   "metadata": {},
   "source": [
    "### 创建集合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97b8fa59",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MilvusClient(\"./10027.db\")\n",
    "\n",
    "client.drop_collection(\"_10027\")\n",
    "\n",
    "client.create_collection(\n",
    "    collection_name=\"_10027\",\n",
    "    dimension=384  # The vectors we will use in this demo has 384 dimensions\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd98e79f",
   "metadata": {},
   "source": [
    "### 分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4291e9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../data/10027.txt\", \"r\", encoding=\"utf-8\") as file:\n",
    "    text = file.read()\n",
    "# 1️⃣ 分块器初始化\n",
    "splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=256,\n",
    "    chunk_overlap=50,\n",
    "    separators=[\"\\n\\n\", \"\\n\", \"。\", \"；\", \"？\", \"！\", \".\", \"!\", \"?\", \"，\", \" \", \"\"]\n",
    ")\n",
    "chunks = splitter.split_text(text)\n",
    "clean_chunks = [chunk.lstrip(\"。！？. \") for chunk in chunks]\n",
    "\n",
    "# 保存分块结果到 txt 文件\n",
    "with open(\"../data/chunks_output.txt\", \"w\", encoding=\"utf-8\") as f:\n",
    "    for i, chunk in enumerate(clean_chunks):\n",
    "        f.write(f\"[Chunk {i+1}]: {chunk}\\n\\n\")\n",
    "\n",
    "print(f\"📌 共分出 {len(clean_chunks)} 个 chunk：\")\n",
    "for i, chunk in enumerate(clean_chunks):\n",
    "    print(f\"[Chunk {i+1}]: {chunk}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78c46cd8",
   "metadata": {},
   "source": [
    "### 编码（bge-small-en-v1.5）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a453da4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SentenceTransformer(\"../model/bge-small-en-v1.5\", device=\"cuda\")\n",
    "\n",
    "vectors = model.encode(chunks)\n",
    "np.save(\"../data/10027_milvus_vectors.npy\", vectors, allow_pickle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee23f17f",
   "metadata": {},
   "source": [
    "### 构建数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68c54a56",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = [[ np.random.uniform(-1, 1) for _ in range(384) ] for _ in range(len(chunks)) ]\n",
    "data = [ {\"id\": i, \"vector\": vectors[i], \"text\": chunks[i]} for i in range(len(vectors)) ]\n",
    "print(f\"📌 数据准备完成，共 {len(data)} 条数据\")\n",
    "# print(type(data))\n",
    "# print(type(data[0]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05fd8406",
   "metadata": {},
   "source": [
    "### 插入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfee9ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 插入数据并检查插入结果\n",
    "batch_size = 10\n",
    "for i in range(0, len(data), batch_size):\n",
    "    batch_data = data[i:i+batch_size]\n",
    "    res = client.insert(\n",
    "        collection_name=\"_10027\",\n",
    "        data=batch_data\n",
    "    )\n",
    "    print(f\"📌 批次插入成功，共插入 {len(res)} 条数据\")\n",
    "\n",
    "# 确保已经插入了足够的文本数据\n",
    "print(\"📌 插入数据完成，共插入条数：\", len(data))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "310e13df",
   "metadata": {},
   "source": [
    "### 查询全部数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59da4503",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = client.query(\n",
    "    collection_name=\"_10027\",\n",
    "    output_fields=[\"text\"],\n",
    "    limit=5,\n",
    ")\n",
    "print(res)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f711271",
   "metadata": {},
   "source": [
    "### 查询"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d39aed6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_vector = model.encode([\"工作时间\"])\n",
    "res = client.search(\n",
    "    collection_name=\"_10027\",\n",
    "    output_fields=[\"text\"],\n",
    "    top_k=5,\n",
    "    data=query_vector,\n",
    ")\n",
    "# print(res)\n",
    "\n",
    "print(\"📌 查询结果：\")\n",
    "for i, result in enumerate(res):\n",
    "    print(f\"[Result {i+1}]: {result}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
